'''
爬取Ajax类型的网页
'''
import requests
import logging
# 使用常规方式访问，获取到的源代码
# url = 'https://spa1.scrape.center/'
# html = requests.get(url).text
# print(html)

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s - %(levelname)s - %(message)s')
INDEX_URL = 'https://spa1.scrape.center/api/movie/?limit={limit}&offset={offset}'

# 爬取页面的通用方法
def scrape_api(url):
       logging.info('scraping %s...', url)
       try:
              response = requests.get(url)
              if response.status_code == 200:
                     return response.json()
              logging.error('get invaild status code %s while scraping %s', response.status_code, url)
       except requests.RequestException:
              logging.error('error occurred while scraping %s', url,exc_info=True)


# 爬取列表页的方法
# 传递页码，构造列表页完整URL
LIMIT = 10
def scrape_index(page):
       url = INDEX_URL.format(limit=LIMIT, offset=LIMIT * (page - 1))
       return scrape_api(url)

# 构造详情页的URL
DETATIL_URL = 'https://spa1.scrape.center/api/movie/{id}/'
def scrape_detail(id):
       url = DETATIL_URL.format(id=id)
       return scrape_api(url)



# 将数据存入MongoDB数据库
MONGO_CONNECTION_STRING = 'mongodb://localhost:27017'
MONGO_DB_NAME = 'moives'
MONGO_COLLECTION_NAME = 'moives'

import pymongo
client = pymongo.MongoClient(MONGO_CONNECTION_STRING)
db = client['moives']
collection = db['moives']

def save_data(data):
       collection.update_one({
              'name': data.get('name')
       }, {
              '$set': data
       }, upsert=True)
TOTAL_PAGE = 10
def main():
       for page in range(1,TOTAL_PAGE + 1):
              index_data = scrape_index(page)
              for item in index_data.get('results'):
                     id = item.get('id')
                     detail_data = scrape_detail(id)
                     logging.info('detail data %s', detail_data)
                     save_data(detail_data)
                     logging.info('scraped page %d', page)


if __name__ == '__main__':
       main()